{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Notebook written by [Zhedong Zheng](https://github.com/zhedongzheng)\n",
    "\n",
    "![alt text](vae.png)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "from modified import ModifiedBasicDecoder, ModifiedBeamSearchDecoder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "PARAMS = {\n",
    "    'max_len': 15,\n",
    "    'vocab_size': 20000,\n",
    "    'embed_dims': 128,\n",
    "    'rnn_size': 128,\n",
    "    'latent_size': 16,\n",
    "    'beam_width': 5,\n",
    "    'clip_norm': 5.0,\n",
    "    'anneal_max': 1.0,\n",
    "    'anneal_bias': 6000,\n",
    "    'batch_size': 128,\n",
    "    'n_epochs': 30,\n",
    "    'word_dropout_rate': 0.8,\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def build_vocab(index_from=4):\n",
    "    PARAMS['word2idx'] = tf.keras.datasets.imdb.get_word_index()\n",
    "    PARAMS['word2idx'] = {k: (v + index_from) for k, v in PARAMS['word2idx'].items()}\n",
    "    PARAMS['word2idx']['<pad>'] = 0\n",
    "    PARAMS['word2idx']['<start>'] = 1\n",
    "    PARAMS['word2idx']['<unk>'] = 2\n",
    "    PARAMS['word2idx']['<end>'] = 3\n",
    "    PARAMS['idx2word'] = {i: w for w, i in PARAMS['word2idx'].items()}\n",
    "\n",
    "    \n",
    "def load_data(index_from=4):\n",
    "    (X_train, _), (X_test, _) = tf.contrib.keras.datasets\\\n",
    "        .imdb.load_data(num_words=PARAMS['vocab_size'], index_from=index_from)\n",
    "    return (X_train, X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def kl_w_fn(global_step):\n",
    "    return PARAMS['anneal_max'] * tf.sigmoid(\n",
    "        (10 / PARAMS['anneal_bias']) * (\n",
    "        tf.to_float(global_step) - tf.constant(PARAMS['anneal_bias'] / 2)))\n",
    "\n",
    "def clip_grads(loss):\n",
    "    variables = tf.trainable_variables()\n",
    "    grads = tf.gradients(loss, variables)\n",
    "    clipped_grads, _ = tf.clip_by_global_norm(grads, PARAMS['clip_norm'])\n",
    "    return zip(clipped_grads, variables)\n",
    "\n",
    "def rnn_cell():\n",
    "    return tf.nn.rnn_cell.GRUCell(PARAMS['rnn_size'],\n",
    "                                  kernel_initializer=tf.orthogonal_initializer())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def forward(inputs, labels, mode):\n",
    "    is_training = (mode == tf.estimator.ModeKeys.TRAIN)\n",
    "    enc_seq_len = tf.count_nonzero(inputs, 1, dtype=tf.int32)\n",
    "    batch_sz = tf.shape(inputs)[0]\n",
    "    \n",
    "    with tf.variable_scope('Encoder'):\n",
    "        embedding = tf.get_variable('lookup_table', [PARAMS['vocab_size'], PARAMS['embed_dims']])\n",
    "        x = tf.nn.embedding_lookup(embedding, inputs)\n",
    "        \n",
    "        _, enc_state = tf.nn.dynamic_rnn(rnn_cell(), x, enc_seq_len, dtype=tf.float32)\n",
    "        \n",
    "        z_mean = tf.layers.dense(enc_state, PARAMS['latent_size'])\n",
    "        z_var = tf.layers.dense(enc_state, PARAMS['latent_size'])\n",
    "        \n",
    "    posterior = tf.contrib.distributions.MultivariateNormalDiag(z_mean, z_var)\n",
    "    prior = tf.contrib.distributions.MultivariateNormalDiag(tf.zeros_like(z_mean),\n",
    "                                                            tf.ones_like(z_var))\n",
    "    z = posterior.sample()\n",
    "        \n",
    "    with tf.variable_scope('Decoder'):\n",
    "        init_state = tf.layers.dense(z, PARAMS['rnn_size'], tf.nn.elu)\n",
    "        output_proj = tf.layers.Dense(PARAMS['vocab_size'])\n",
    "        dec_cell = rnn_cell()\n",
    "        \n",
    "        if is_training:\n",
    "            dec_seq_len = tf.count_nonzero(labels['dec_out'], 1, dtype=tf.int32)\n",
    "            \n",
    "            helper = tf.contrib.seq2seq.TrainingHelper(\n",
    "                inputs = tf.nn.embedding_lookup(embedding, labels['dec_inp']),\n",
    "                sequence_length = dec_seq_len)\n",
    "            decoder = ModifiedBasicDecoder(\n",
    "                cell = dec_cell,\n",
    "                helper = helper,\n",
    "                initial_state = init_state,\n",
    "                output_layer = output_proj,\n",
    "                concat_z = z)\n",
    "            decoder_output, _, _ = tf.contrib.seq2seq.dynamic_decode(\n",
    "                decoder = decoder,\n",
    "                maximum_iterations = tf.reduce_max(dec_seq_len))\n",
    "            logits = decoder_output.rnn_output\n",
    "            \n",
    "            return logits, posterior, prior\n",
    "        else:\n",
    "            tiled_z = tf.tile(tf.expand_dims(z, 1), [1, PARAMS['beam_width'], 1])\n",
    "            \n",
    "            decoder = ModifiedBeamSearchDecoder(\n",
    "                cell = dec_cell,\n",
    "                embedding = embedding,\n",
    "                start_tokens = tf.tile(tf.constant([PARAMS['word2idx']['<start>']], tf.int32),\n",
    "                                       [batch_sz]),\n",
    "                end_token = PARAMS['word2idx']['<end>'],\n",
    "                initial_state = tf.contrib.seq2seq.tile_batch(init_state, PARAMS['beam_width']),\n",
    "                beam_width = PARAMS['beam_width'],\n",
    "                output_layer = output_proj,\n",
    "                concat_z = tiled_z)\n",
    "            decoder_output, _, _ = tf.contrib.seq2seq.dynamic_decode(\n",
    "                decoder = decoder)\n",
    "            \n",
    "            return decoder_output.predicted_ids[:, :, 0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def model_fn(features, labels, mode):\n",
    "    logits_or_ids = forward(features, labels, mode)        \n",
    "    \n",
    "    if mode == tf.estimator.ModeKeys.PREDICT:\n",
    "        return tf.estimator.EstimatorSpec(mode, predictions=logits_or_ids)\n",
    "        \n",
    "    if mode == tf.estimator.ModeKeys.TRAIN:\n",
    "        logits, posterior, prior = logits_or_ids\n",
    "        \n",
    "        out_dist = tf.distributions.Categorical(logits)\n",
    "        \n",
    "        global_step = tf.train.get_global_step()\n",
    "        \n",
    "        nll_loss = - tf.reduce_sum(out_dist.log_prob(labels['dec_out']))\n",
    "        \n",
    "        kl_w = kl_w_fn(global_step)\n",
    "        \n",
    "        kl_loss = tf.reduce_sum(tf.distributions.kl_divergence(posterior, prior))\n",
    "        \n",
    "        loss_op = nll_loss + kl_w * kl_loss\n",
    "        \n",
    "        train_op = tf.train.AdamOptimizer().apply_gradients(clip_grads(loss_op),\n",
    "                                                            global_step = global_step)\n",
    "        \n",
    "        lth = tf.train.LoggingTensorHook(\n",
    "            {'nll_loss': nll_loss, 'kl_w': kl_w, 'kl_loss': kl_loss}, every_n_iter=100)\n",
    "        \n",
    "        return tf.estimator.EstimatorSpec(\n",
    "            mode=mode, loss=loss_op, train_op=train_op, training_hooks=[lth])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def inf_inp(test_strs):\n",
    "    x = [[PARAMS['word2idx'].get(w, 2) for w in s.split()] for s in test_strs]\n",
    "    x = tf.keras.preprocessing.sequence.pad_sequences(\n",
    "        x, PARAMS['max_len'], truncating='post', padding='post')\n",
    "    return x\n",
    "\n",
    "\n",
    "def demo(test_strs, pred_ids):\n",
    "    for s, pred in zip(test_strs, pred_ids):\n",
    "        print('\\nOriginal:', s)\n",
    "        print('Reconstr:', ' '.join([PARAMS['idx2word'].get(idx, '<unk>') for idx in pred]))\n",
    "\n",
    "        \n",
    "def word_dropout(x):\n",
    "    is_dropped = np.random.binomial(1, PARAMS['word_dropout_rate'], x.shape)\n",
    "    fn = np.vectorize(lambda x, k: PARAMS['word2idx']['<unk>'] if (\n",
    "                      k and (x not in range(4))) else x)\n",
    "    return fn(x, is_dropped)\n",
    "        \n",
    "        \n",
    "def main():\n",
    "    build_vocab()\n",
    "    \n",
    "    X = np.concatenate(load_data())\n",
    "    X = np.concatenate((tf.keras.preprocessing.sequence.pad_sequences(\n",
    "                            X, PARAMS['max_len'], truncating='post', padding='post'),\n",
    "                        tf.keras.preprocessing.sequence.pad_sequences(\n",
    "                            X, PARAMS['max_len'], truncating='pre', padding='post')))\n",
    "\n",
    "    enc_inp = X[:, 1:]\n",
    "    dec_inp = X[:]\n",
    "    dec_out = np.concatenate([X[:, 1:], np.full([X.shape[0], 1], PARAMS['word2idx']['<end>'])], 1)\n",
    "\n",
    "    test_strs = ['i love this film and i think it is one of the best films',\n",
    "                 'this movie is a waste of time and there is no point to watch it']\n",
    "\n",
    "    estimator = tf.estimator.Estimator(model_fn)\n",
    "\n",
    "    for i in range(PARAMS['n_epochs']):\n",
    "        print('Epoch %d/%d'%(i+1, PARAMS['n_epochs']))\n",
    "        print()\n",
    "        \n",
    "        estimator.train(tf.estimator.inputs.numpy_input_fn(\n",
    "            x = enc_inp,\n",
    "            y = {'dec_inp': word_dropout(dec_inp), 'dec_out': dec_out},\n",
    "            batch_size = PARAMS['batch_size'],\n",
    "            shuffle = True))\n",
    "\n",
    "        pred_ids = list(estimator.predict(tf.estimator.inputs.numpy_input_fn(\n",
    "            x = inf_inp(test_strs),\n",
    "            shuffle = False)))\n",
    "\n",
    "        demo(test_strs, pred_ids)\n",
    "        print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Using default config.\n",
      "WARNING:tensorflow:Using temporary folder as model directory: /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpqgwokls5\n",
      "INFO:tensorflow:Using config: {'_model_dir': '/var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpqgwokls5', '_tf_random_seed': None, '_save_summary_steps': 100, '_save_checkpoints_steps': None, '_save_checkpoints_secs': 600, '_session_config': None, '_keep_checkpoint_max': 5, '_keep_checkpoint_every_n_hours': 10000, '_log_step_count_steps': 100, '_train_distribute': None, '_service': None, '_cluster_spec': <tensorflow.python.training.server_lib.ClusterSpec object at 0x123f16eb8>, '_task_type': 'worker', '_task_id': 0, '_global_id_in_cluster': 0, '_master': '', '_evaluation_master': '', '_is_chief': True, '_num_ps_replicas': 0, '_num_worker_replicas': 1}\n",
      "Epoch 1/30\n",
      "\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 1 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpqgwokls5/model.ckpt.\n",
      "INFO:tensorflow:loss = 19092.125, step = 1\n",
      "INFO:tensorflow:nll_loss = 19014.932, kl_w = 0.006692851, kl_loss = 11533.825\n",
      "INFO:tensorflow:global_step/sec: 0.748254\n",
      "INFO:tensorflow:loss = 12374.434, step = 101 (133.647 sec)\n",
      "INFO:tensorflow:nll_loss = 12366.619, kl_w = 0.007897083, kl_loss = 989.49225 (133.646 sec)\n",
      "INFO:tensorflow:global_step/sec: 0.71286\n",
      "INFO:tensorflow:loss = 11896.239, step = 201 (140.279 sec)\n",
      "INFO:tensorflow:nll_loss = 11844.752, kl_w = 0.009315956, kl_loss = 5526.762 (140.279 sec)\n",
      "INFO:tensorflow:global_step/sec: 0.722575\n",
      "INFO:tensorflow:loss = 11563.3125, step = 301 (138.394 sec)\n",
      "INFO:tensorflow:nll_loss = 11500.285, kl_w = 0.010986943, kl_loss = 5736.5547 (138.394 sec)\n",
      "INFO:tensorflow:global_step/sec: 0.756242\n",
      "INFO:tensorflow:loss = 11410.647, step = 401 (132.233 sec)\n",
      "INFO:tensorflow:nll_loss = 11352.627, kl_w = 0.012953726, kl_loss = 4479.048 (132.233 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 441 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpqgwokls5/model.ckpt.\n",
      "INFO:tensorflow:global_step/sec: 0.750762\n",
      "INFO:tensorflow:loss = 11223.874, step = 501 (133.199 sec)\n",
      "INFO:tensorflow:nll_loss = 11171.408, kl_w = 0.015267149, kl_loss = 3436.4924 (133.199 sec)\n",
      "INFO:tensorflow:global_step/sec: 0.764738\n",
      "INFO:tensorflow:loss = 11683.915, step = 601 (130.763 sec)\n",
      "INFO:tensorflow:nll_loss = 11631.119, kl_w = 0.01798621, kl_loss = 2935.3677 (130.763 sec)\n",
      "INFO:tensorflow:global_step/sec: 0.745801\n",
      "INFO:tensorflow:loss = 11653.028, step = 701 (134.084 sec)\n",
      "INFO:tensorflow:nll_loss = 11594.348, kl_w = 0.021179108, kl_loss = 2770.684 (134.084 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 782 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpqgwokls5/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 2871.7524.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpqgwokls5/model.ckpt-782\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "\n",
      "Original: i love this film and i think it is one of the best films\n",
      "Reconstr: this this this this this this this this a a a the the <unk> <end>\n",
      "\n",
      "Original: this movie is a waste of time and there is no point to watch it\n",
      "Reconstr: br br br br br br br br br br br br br <end> <end>\n",
      "\n",
      "Epoch 2/30\n",
      "\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpqgwokls5/model.ckpt-782\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 783 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpqgwokls5/model.ckpt.\n",
      "INFO:tensorflow:loss = 11361.067, step = 783\n",
      "INFO:tensorflow:nll_loss = 11294.637, kl_w = 0.024205629, kl_loss = 2744.4307\n",
      "INFO:tensorflow:global_step/sec: 0.774442\n",
      "INFO:tensorflow:loss = 11322.495, step = 883 (129.126 sec)\n",
      "INFO:tensorflow:nll_loss = 11233.264, kl_w = 0.028470589, kl_loss = 3134.1646 (129.126 sec)\n",
      "INFO:tensorflow:global_step/sec: 0.72816\n",
      "INFO:tensorflow:loss = 11153.089, step = 983 (137.333 sec)\n",
      "INFO:tensorflow:nll_loss = 11042.942, kl_w = 0.033461247, kl_loss = 3291.7654 (137.333 sec)\n",
      "INFO:tensorflow:global_step/sec: 0.739537\n",
      "INFO:tensorflow:loss = 10637.993, step = 1083 (135.220 sec)\n",
      "INFO:tensorflow:nll_loss = 10503.816, kl_w = 0.039291352, kl_loss = 3414.916 (135.220 sec)\n",
      "INFO:tensorflow:global_step/sec: 0.707756\n",
      "INFO:tensorflow:loss = 10280.8545, step = 1183 (141.293 sec)\n",
      "INFO:tensorflow:nll_loss = 10124.744, kl_w = 0.04608883, kl_loss = 3387.1523 (141.292 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 1220 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpqgwokls5/model.ckpt.\n",
      "INFO:tensorflow:global_step/sec: 0.667831\n",
      "INFO:tensorflow:loss = 11090.775, step = 1283 (149.738 sec)\n",
      "INFO:tensorflow:nll_loss = 10948.215, kl_w = 0.053996176, kl_loss = 2640.1997 (149.738 sec)\n",
      "INFO:tensorflow:global_step/sec: 0.644689\n",
      "INFO:tensorflow:loss = 10970.904, step = 1383 (155.113 sec)\n",
      "INFO:tensorflow:nll_loss = 10798.908, kl_w = 0.06317033, kl_loss = 2722.7388 (155.114 sec)\n",
      "INFO:tensorflow:global_step/sec: 0.697896\n",
      "INFO:tensorflow:loss = 10768.245, step = 1483 (143.288 sec)\n",
      "INFO:tensorflow:nll_loss = 10566.2295, kl_w = 0.07378165, kl_loss = 2738.0146 (143.288 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 1564 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpqgwokls5/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 2706.0225.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpqgwokls5/model.ckpt-1564\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "\n",
      "Original: i love this film and i think it is one of the best films\n",
      "Reconstr: i saw this movie movie i i seen this this of of of of <end>\n",
      "\n",
      "Original: this movie is a waste of time and there is no point to watch it\n",
      "Reconstr: this movie is of of of and and i seen it it a time <end>\n",
      "\n",
      "Epoch 3/30\n",
      "\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpqgwokls5/model.ckpt-1564\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 1565 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpqgwokls5/model.ckpt.\n",
      "INFO:tensorflow:loss = 10689.293, step = 1565\n",
      "INFO:tensorflow:nll_loss = 10432.85, kl_w = 0.08368247, kl_loss = 3064.482\n",
      "INFO:tensorflow:global_step/sec: 0.631525\n",
      "INFO:tensorflow:loss = 10130.908, step = 1665 (158.348 sec)\n",
      "INFO:tensorflow:nll_loss = 9849.811, kl_w = 0.09738124, kl_loss = 2886.565 (158.348 sec)\n",
      "INFO:tensorflow:global_step/sec: 0.649609\n",
      "INFO:tensorflow:loss = 10161.426, step = 1765 (153.939 sec)\n",
      "INFO:tensorflow:nll_loss = 9847.902, kl_w = 0.11304584, kl_loss = 2773.4214 (153.939 sec)\n",
      "INFO:tensorflow:global_step/sec: 0.718717\n",
      "INFO:tensorflow:loss = 10579.439, step = 1865 (139.137 sec)\n",
      "INFO:tensorflow:nll_loss = 10247.537, kl_w = 0.13086486, kl_loss = 2536.221 (139.137 sec)\n",
      "INFO:tensorflow:global_step/sec: 0.723951\n",
      "INFO:tensorflow:loss = 10633.033, step = 1965 (138.131 sec)\n",
      "INFO:tensorflow:nll_loss = 10265.242, kl_w = 0.15101443, kl_loss = 2435.4692 (138.131 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 1971 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpqgwokls5/model.ckpt.\n",
      "INFO:tensorflow:global_step/sec: 0.709954\n",
      "INFO:tensorflow:loss = 10242.295, step = 2065 (140.854 sec)\n",
      "INFO:tensorflow:nll_loss = 9810.689, kl_w = 0.17364664, kl_loss = 2485.5413 (140.854 sec)\n",
      "INFO:tensorflow:global_step/sec: 0.664997\n",
      "INFO:tensorflow:loss = 10856.193, step = 2165 (150.377 sec)\n",
      "INFO:tensorflow:nll_loss = 10408.703, kl_w = 0.19887616, kl_loss = 2250.0974 (150.378 sec)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:global_step/sec: 0.71162\n",
      "INFO:tensorflow:loss = 10165.197, step = 2265 (140.524 sec)\n",
      "INFO:tensorflow:nll_loss = 9637.449, kl_w = 0.22676536, kl_loss = 2327.2856 (140.524 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 2346 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpqgwokls5/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 2470.2595.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpqgwokls5/model.ckpt-2346\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "\n",
      "Original: i love this film and i think it is one of the best films\n",
      "Reconstr: i was this movie i have seen this movie of the film of the <end>\n",
      "\n",
      "Original: this movie is a waste of time and there is no point to watch it\n",
      "Reconstr: this is a lot of the movie to not the time it it it <end>\n",
      "\n",
      "Epoch 4/30\n",
      "\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpqgwokls5/model.ckpt-2346\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 2347 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpqgwokls5/model.ckpt.\n",
      "INFO:tensorflow:loss = 9878.03, step = 2347\n",
      "INFO:tensorflow:nll_loss = 9319.192, kl_w = 0.25161827, kl_loss = 2220.9736\n",
      "INFO:tensorflow:global_step/sec: 0.739995\n",
      "INFO:tensorflow:loss = 9895.965, step = 2447 (135.137 sec)\n",
      "INFO:tensorflow:nll_loss = 9281.529, kl_w = 0.2842792, kl_loss = 2161.3792 (135.137 sec)\n",
      "INFO:tensorflow:global_step/sec: 0.720302\n",
      "INFO:tensorflow:loss = 10501.709, step = 2547 (138.831 sec)\n",
      "INFO:tensorflow:nll_loss = 9874.396, kl_w = 0.31937042, kl_loss = 1964.2169 (138.831 sec)\n",
      "INFO:tensorflow:global_step/sec: 0.71002\n",
      "INFO:tensorflow:loss = 10813.224, step = 2647 (140.841 sec)\n",
      "INFO:tensorflow:nll_loss = 10155.152, kl_w = 0.35663486, kl_loss = 1845.2235 (140.841 sec)\n",
      "INFO:tensorflow:global_step/sec: 0.736094\n",
      "INFO:tensorflow:loss = 10455.315, step = 2747 (135.852 sec)\n",
      "INFO:tensorflow:nll_loss = 9771.088, kl_w = 0.39571938, kl_loss = 1729.0729 (135.852 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 2782 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpqgwokls5/model.ckpt.\n",
      "INFO:tensorflow:global_step/sec: 0.749317\n",
      "INFO:tensorflow:loss = 10689.476, step = 2847 (133.455 sec)\n",
      "INFO:tensorflow:nll_loss = 9964.865, kl_w = 0.43618327, kl_loss = 1661.252 (133.455 sec)\n",
      "INFO:tensorflow:global_step/sec: 0.788089\n",
      "INFO:tensorflow:loss = 10322.026, step = 2947 (126.889 sec)\n",
      "INFO:tensorflow:nll_loss = 9570.933, kl_w = 0.47751516, kl_loss = 1572.922 (126.889 sec)\n",
      "INFO:tensorflow:global_step/sec: 0.771859\n",
      "INFO:tensorflow:loss = 10679.094, step = 3047 (129.557 sec)\n",
      "INFO:tensorflow:nll_loss = 9901.791, kl_w = 0.5191573, kl_loss = 1497.2402 (129.558 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 3128 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpqgwokls5/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 2606.5642.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpqgwokls5/model.ckpt-3128\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "\n",
      "Original: i love this film and i think it is one of the best films\n",
      "Reconstr: i loved this movie i have seen it is one of the best of <end>\n",
      "\n",
      "Original: this movie is a waste of time and there is no point to watch it\n",
      "Reconstr: this is one of those movies but not not not much but it <end> <end>\n",
      "\n",
      "Epoch 5/30\n",
      "\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpqgwokls5/model.ckpt-3128\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 3129 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpqgwokls5/model.ckpt.\n",
      "INFO:tensorflow:loss = 10066.756, step = 3129\n",
      "INFO:tensorflow:nll_loss = 9237.309, kl_w = 0.55313194, kl_loss = 1499.5472\n",
      "INFO:tensorflow:global_step/sec: 0.721811\n",
      "INFO:tensorflow:loss = 10233.293, step = 3229 (138.542 sec)\n",
      "INFO:tensorflow:nll_loss = 9351.939, kl_w = 0.5938731, kl_loss = 1484.0767 (138.542 sec)\n",
      "INFO:tensorflow:global_step/sec: 0.769872\n",
      "INFO:tensorflow:loss = 10597.739, step = 3329 (129.891 sec)\n",
      "INFO:tensorflow:nll_loss = 9672.883, kl_w = 0.6333619, kl_loss = 1460.2341 (129.891 sec)\n",
      "INFO:tensorflow:global_step/sec: 0.783436\n",
      "INFO:tensorflow:loss = 10652.835, step = 3429 (127.643 sec)\n",
      "INFO:tensorflow:nll_loss = 9698.25, kl_w = 0.6711373, kl_loss = 1422.3391 (127.643 sec)\n",
      "INFO:tensorflow:global_step/sec: 0.781999\n",
      "INFO:tensorflow:loss = 10833.713, step = 3529 (127.877 sec)\n",
      "INFO:tensorflow:nll_loss = 9906.961, kl_w = 0.7068222, kl_loss = 1311.1526 (127.877 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 3587 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpqgwokls5/model.ckpt.\n",
      "INFO:tensorflow:global_step/sec: 0.773453\n",
      "INFO:tensorflow:loss = 10693.083, step = 3629 (129.290 sec)\n",
      "INFO:tensorflow:nll_loss = 9773.248, kl_w = 0.7401343, kl_loss = 1242.7942 (129.290 sec)\n",
      "INFO:tensorflow:global_step/sec: 0.785222\n",
      "INFO:tensorflow:loss = 10317.069, step = 3729 (127.352 sec)\n",
      "INFO:tensorflow:nll_loss = 9348.539, kl_w = 0.7708882, kl_loss = 1256.3823 (127.353 sec)\n",
      "INFO:tensorflow:global_step/sec: 0.770887\n",
      "INFO:tensorflow:loss = 10374.451, step = 3829 (129.721 sec)\n",
      "INFO:tensorflow:nll_loss = 9372.504, kl_w = 0.79899096, kl_loss = 1254.0151 (129.721 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 3910 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpqgwokls5/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 2507.4297.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpqgwokls5/model.ckpt-3910\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "\n",
      "Original: i love this film and i think it is one of the best films\n",
      "Reconstr: i love this movie and i have seen this one of the best of <end>\n",
      "\n",
      "Original: this movie is a waste of time and there is no point to watch it\n",
      "Reconstr: this is one of those movies that i have seen the time but i <end>\n",
      "\n",
      "Epoch 6/30\n",
      "\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpqgwokls5/model.ckpt-3910\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 3911 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpqgwokls5/model.ckpt.\n",
      "INFO:tensorflow:loss = 11004.065, step = 3911\n",
      "INFO:tensorflow:nll_loss = 9980.787, kl_w = 0.82004714, kl_loss = 1247.8284\n",
      "INFO:tensorflow:global_step/sec: 0.779908\n",
      "INFO:tensorflow:loss = 10630.06, step = 4011 (128.222 sec)\n",
      "INFO:tensorflow:nll_loss = 9636.911, kl_w = 0.84334546, kl_loss = 1177.6293 (128.221 sec)\n",
      "INFO:tensorflow:global_step/sec: 0.795308\n",
      "INFO:tensorflow:loss = 10563.314, step = 4111 (125.737 sec)\n",
      "INFO:tensorflow:nll_loss = 9524.352, kl_w = 0.8641271, kl_loss = 1202.3264 (125.737 sec)\n",
      "INFO:tensorflow:global_step/sec: 0.758835\n",
      "INFO:tensorflow:loss = 10221.637, step = 4211 (131.781 sec)\n",
      "INFO:tensorflow:nll_loss = 9222.384, kl_w = 0.88253593, kl_loss = 1132.2516 (131.781 sec)\n",
      "INFO:tensorflow:global_step/sec: 0.776936\n",
      "INFO:tensorflow:loss = 10296.43, step = 4311 (128.711 sec)\n",
      "INFO:tensorflow:nll_loss = 9230.016, kl_w = 0.89874285, kl_loss = 1186.5616 (128.711 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 4375 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpqgwokls5/model.ckpt.\n",
      "INFO:tensorflow:global_step/sec: 0.719737\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:loss = 11141.858, step = 4411 (138.940 sec)\n",
      "INFO:tensorflow:nll_loss = 10165.846, kl_w = 0.9129343, kl_loss = 1069.0938 (138.940 sec)\n",
      "INFO:tensorflow:global_step/sec: 0.777566\n",
      "INFO:tensorflow:loss = 10777.204, step = 4511 (128.606 sec)\n",
      "INFO:tensorflow:nll_loss = 9790.451, kl_w = 0.92530197, kl_loss = 1066.4117 (128.606 sec)\n",
      "INFO:tensorflow:global_step/sec: 0.756028\n",
      "INFO:tensorflow:loss = 11016.194, step = 4611 (132.270 sec)\n",
      "INFO:tensorflow:nll_loss = 10036.266, kl_w = 0.93603605, kl_loss = 1046.8916 (132.271 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 4692 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpqgwokls5/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 2645.3716.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpqgwokls5/model.ckpt-4692\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "\n",
      "Original: i love this film and i think it is one of the best films\n",
      "Reconstr: i have seen this movie and i give it out of the best films <end>\n",
      "\n",
      "Original: this movie is a waste of time and there is no point to watch it\n",
      "Reconstr: if you is a lot of this movie to the time to see it <end>\n",
      "\n",
      "Epoch 7/30\n",
      "\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpqgwokls5/model.ckpt-4692\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 4693 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpqgwokls5/model.ckpt.\n",
      "INFO:tensorflow:loss = 10503.174, step = 4693\n",
      "INFO:tensorflow:nll_loss = 9479.979, kl_w = 0.94374704, kl_loss = 1084.1838\n",
      "INFO:tensorflow:global_step/sec: 0.744421\n",
      "INFO:tensorflow:loss = 10847.034, step = 4793 (134.334 sec)\n",
      "INFO:tensorflow:nll_loss = 9858.21, kl_w = 0.95196813, kl_loss = 1038.7153 (134.334 sec)\n",
      "INFO:tensorflow:global_step/sec: 0.712906\n",
      "INFO:tensorflow:loss = 10729.833, step = 4893 (140.271 sec)\n",
      "INFO:tensorflow:nll_loss = 9736.208, kl_w = 0.9590399, kl_loss = 1036.0624 (140.271 sec)\n",
      "INFO:tensorflow:global_step/sec: 0.720469\n",
      "INFO:tensorflow:loss = 10934.033, step = 4993 (138.799 sec)\n",
      "INFO:tensorflow:nll_loss = 9946.64, kl_w = 0.9651086, kl_loss = 1023.0906 (138.799 sec)\n",
      "INFO:tensorflow:global_step/sec: 0.72834\n",
      "INFO:tensorflow:loss = 10984.219, step = 5093 (137.299 sec)\n",
      "INFO:tensorflow:nll_loss = 9957.238, kl_w = 0.97030604, kl_loss = 1058.4087 (137.298 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 5128 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpqgwokls5/model.ckpt.\n",
      "INFO:tensorflow:global_step/sec: 0.711546\n",
      "INFO:tensorflow:loss = 10300.959, step = 5193 (140.539 sec)\n",
      "INFO:tensorflow:nll_loss = 9267.202, kl_w = 0.97474945, kl_loss = 1060.5364 (140.539 sec)\n",
      "INFO:tensorflow:global_step/sec: 0.718583\n",
      "INFO:tensorflow:loss = 10498.09, step = 5293 (139.163 sec)\n",
      "INFO:tensorflow:nll_loss = 9451.598, kl_w = 0.9785427, kl_loss = 1069.44 (139.163 sec)\n",
      "INFO:tensorflow:global_step/sec: 0.784283\n",
      "INFO:tensorflow:loss = 10153.263, step = 5393 (127.505 sec)\n",
      "INFO:tensorflow:nll_loss = 9059.229, kl_w = 0.9817768, kl_loss = 1114.3408 (127.505 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 5474 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpqgwokls5/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 2558.79.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpqgwokls5/model.ckpt-5474\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "\n",
      "Original: i love this film and i think it is one of the best films\n",
      "Reconstr: i love this movie and i give this one of the best films br <end>\n",
      "\n",
      "Original: this movie is a waste of time and there is no point to watch it\n",
      "Reconstr: this movie is one of those movies to watch a few time to watch <end>\n",
      "\n",
      "Epoch 8/30\n",
      "\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpqgwokls5/model.ckpt-5474\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 5475 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpqgwokls5/model.ckpt.\n",
      "INFO:tensorflow:loss = 9843.156, step = 5475\n",
      "INFO:tensorflow:nll_loss = 8801.59, kl_w = 0.98406756, kl_loss = 1058.4294\n",
      "INFO:tensorflow:global_step/sec: 0.774335\n",
      "INFO:tensorflow:loss = 11029.208, step = 5575 (129.144 sec)\n",
      "INFO:tensorflow:nll_loss = 9989.068, kl_w = 0.9864804, kl_loss = 1054.395 (129.144 sec)\n",
      "INFO:tensorflow:global_step/sec: 0.722494\n",
      "INFO:tensorflow:loss = 10769.403, step = 5675 (138.409 sec)\n",
      "INFO:tensorflow:nll_loss = 9725.258, kl_w = 0.98853207, kl_loss = 1056.259 (138.409 sec)\n",
      "INFO:tensorflow:global_step/sec: 0.777138\n",
      "INFO:tensorflow:loss = 10958.06, step = 5775 (128.677 sec)\n",
      "INFO:tensorflow:nll_loss = 9946.535, kl_w = 0.9902755, kl_loss = 1021.4574 (128.677 sec)\n",
      "INFO:tensorflow:global_step/sec: 0.751862\n",
      "INFO:tensorflow:loss = 10637.767, step = 5875 (133.004 sec)\n",
      "INFO:tensorflow:nll_loss = 9593.6455, kl_w = 0.9917561, kl_loss = 1052.8003 (133.004 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 5920 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpqgwokls5/model.ckpt.\n",
      "INFO:tensorflow:global_step/sec: 0.634386\n",
      "INFO:tensorflow:loss = 10754.754, step = 5975 (157.632 sec)\n",
      "INFO:tensorflow:nll_loss = 9713.189, kl_w = 0.99301285, kl_loss = 1048.8928 (157.632 sec)\n",
      "INFO:tensorflow:global_step/sec: 0.748744\n",
      "INFO:tensorflow:loss = 10238.15, step = 6075 (133.557 sec)\n",
      "INFO:tensorflow:nll_loss = 9149.72, kl_w = 0.9940791, kl_loss = 1094.9138 (133.557 sec)\n",
      "INFO:tensorflow:global_step/sec: 0.750346\n",
      "INFO:tensorflow:loss = 10405.38, step = 6175 (133.272 sec)\n",
      "INFO:tensorflow:nll_loss = 9282.02, kl_w = 0.99498355, kl_loss = 1129.0244 (133.272 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 6256 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpqgwokls5/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 2648.9255.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpqgwokls5/model.ckpt-6256\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "\n",
      "Original: i love this film and i think it is one of the best films\n",
      "Reconstr: this is the worst movie i have ever seen this one of the best <end>\n",
      "\n",
      "Original: this movie is a waste of time and there is no point to watch it\n",
      "Reconstr: this is a waste of those movies i've ever seen if you don't know <end>\n",
      "\n",
      "Epoch 9/30\n",
      "\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpqgwokls5/model.ckpt-6256\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 6257 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpqgwokls5/model.ckpt.\n",
      "INFO:tensorflow:loss = 11240.399, step = 6257\n",
      "INFO:tensorflow:nll_loss = 10119.873, kl_w = 0.9956215, kl_loss = 1125.4546\n",
      "INFO:tensorflow:global_step/sec: 0.744832\n",
      "INFO:tensorflow:loss = 10730.914, step = 6357 (134.260 sec)\n",
      "INFO:tensorflow:nll_loss = 9683.884, kl_w = 0.9962913, kl_loss = 1050.9277 (134.260 sec)\n",
      "INFO:tensorflow:global_step/sec: 0.725868\n",
      "INFO:tensorflow:loss = 10755.335, step = 6457 (137.768 sec)\n",
      "INFO:tensorflow:nll_loss = 9686.5205, kl_w = 0.99685884, kl_loss = 1072.1827 (137.768 sec)\n",
      "INFO:tensorflow:global_step/sec: 0.718696\n",
      "INFO:tensorflow:loss = 10429.697, step = 6557 (139.139 sec)\n",
      "INFO:tensorflow:nll_loss = 9304.272, kl_w = 0.9973398, kl_loss = 1128.4268 (139.139 sec)\n",
      "INFO:tensorflow:global_step/sec: 0.698485\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:loss = 10160.389, step = 6657 (143.167 sec)\n",
      "INFO:tensorflow:nll_loss = 9010.473, kl_w = 0.99774724, kl_loss = 1152.5125 (143.167 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 6690 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpqgwokls5/model.ckpt.\n",
      "INFO:tensorflow:global_step/sec: 0.723974\n",
      "INFO:tensorflow:loss = 10228.971, step = 6757 (138.127 sec)\n",
      "INFO:tensorflow:nll_loss = 9054.753, kl_w = 0.99809235, kl_loss = 1176.4619 (138.126 sec)\n",
      "INFO:tensorflow:global_step/sec: 0.703937\n",
      "INFO:tensorflow:loss = 10054.073, step = 6857 (142.058 sec)\n",
      "INFO:tensorflow:nll_loss = 8881.458, kl_w = 0.99838483, kl_loss = 1174.5118 (142.058 sec)\n",
      "INFO:tensorflow:global_step/sec: 0.699849\n",
      "INFO:tensorflow:loss = 10893.012, step = 6957 (142.888 sec)\n",
      "INFO:tensorflow:nll_loss = 9842.256, kl_w = 0.9986324, kl_loss = 1052.1951 (142.888 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 7038 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpqgwokls5/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 2716.4707.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpqgwokls5/model.ckpt-7038\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "\n",
      "Original: i love this film and i think it is one of the best films\n",
      "Reconstr: i love this film and i have ever seen it is a great of <end>\n",
      "\n",
      "Original: this movie is a waste of time and there is no point to watch it\n",
      "Reconstr: this is a waste of time to waste your time to watch this movie <end>\n",
      "\n",
      "Epoch 10/30\n",
      "\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpqgwokls5/model.ckpt-7038\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 7039 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpqgwokls5/model.ckpt.\n",
      "INFO:tensorflow:loss = 10705.874, step = 7039\n",
      "INFO:tensorflow:nll_loss = 9592.941, kl_w = 0.99880695, kl_loss = 1114.2618\n",
      "INFO:tensorflow:global_step/sec: 0.738628\n",
      "INFO:tensorflow:loss = 10107.352, step = 7139 (135.387 sec)\n",
      "INFO:tensorflow:nll_loss = 8933.889, kl_w = 0.9989899, kl_loss = 1174.6489 (135.387 sec)\n",
      "INFO:tensorflow:global_step/sec: 0.68246\n",
      "INFO:tensorflow:loss = 10084.328, step = 7239 (146.529 sec)\n",
      "INFO:tensorflow:nll_loss = 8931.982, kl_w = 0.9991448, kl_loss = 1153.3323 (146.529 sec)\n",
      "INFO:tensorflow:global_step/sec: 0.697083\n",
      "INFO:tensorflow:loss = 10010.01, step = 7339 (143.455 sec)\n",
      "INFO:tensorflow:nll_loss = 8833.343, kl_w = 0.999276, kl_loss = 1177.5194 (143.455 sec)\n",
      "INFO:tensorflow:global_step/sec: 0.7036\n",
      "INFO:tensorflow:loss = 10868.754, step = 7439 (142.126 sec)\n",
      "INFO:tensorflow:nll_loss = 9850.707, kl_w = 0.999387, kl_loss = 1018.6713 (142.126 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 7460 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpqgwokls5/model.ckpt.\n",
      "INFO:tensorflow:global_step/sec: 0.689763\n",
      "INFO:tensorflow:loss = 10760.467, step = 7539 (144.977 sec)\n",
      "INFO:tensorflow:nll_loss = 9630.945, kl_w = 0.99948114, kl_loss = 1130.1084 (144.977 sec)\n",
      "INFO:tensorflow:global_step/sec: 0.693318\n",
      "INFO:tensorflow:loss = 10671.129, step = 7639 (144.234 sec)\n",
      "INFO:tensorflow:nll_loss = 9578.532, kl_w = 0.9995608, kl_loss = 1093.0767 (144.234 sec)\n",
      "INFO:tensorflow:global_step/sec: 0.728434\n",
      "INFO:tensorflow:loss = 10674.332, step = 7739 (137.281 sec)\n",
      "INFO:tensorflow:nll_loss = 9576.049, kl_w = 0.9996282, kl_loss = 1098.6919 (137.281 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 7820 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpqgwokls5/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 2510.023.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpqgwokls5/model.ckpt-7820\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "\n",
      "Original: i love this film and i think it is one of the best films\n",
      "Reconstr: i loved this movie and i have been released on the first of the <end>\n",
      "\n",
      "Original: this movie is a waste of time and there is no point to watch it\n",
      "Reconstr: this is one of the worst movies i've seen the time i have seen <end>\n",
      "\n",
      "Epoch 11/30\n",
      "\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpqgwokls5/model.ckpt-7820\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 7821 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpqgwokls5/model.ckpt.\n",
      "INFO:tensorflow:loss = 10833.07, step = 7821\n",
      "INFO:tensorflow:nll_loss = 9690.849, kl_w = 0.99967563, kl_loss = 1142.5918\n",
      "INFO:tensorflow:global_step/sec: 0.752047\n",
      "INFO:tensorflow:loss = 10747.09, step = 7921 (132.971 sec)\n",
      "INFO:tensorflow:nll_loss = 9604.721, kl_w = 0.9997254, kl_loss = 1142.6833 (132.971 sec)\n",
      "INFO:tensorflow:global_step/sec: 0.733365\n",
      "INFO:tensorflow:loss = 10767.288, step = 8021 (136.358 sec)\n",
      "INFO:tensorflow:nll_loss = 9618.032, kl_w = 0.9997676, kl_loss = 1149.5227 (136.358 sec)\n",
      "INFO:tensorflow:global_step/sec: 0.716441\n",
      "INFO:tensorflow:loss = 10839.678, step = 8121 (139.579 sec)\n",
      "INFO:tensorflow:nll_loss = 9738.127, kl_w = 0.99980325, kl_loss = 1101.7673 (139.579 sec)\n",
      "INFO:tensorflow:global_step/sec: 0.700558\n",
      "INFO:tensorflow:loss = 10436.374, step = 8221 (142.744 sec)\n",
      "INFO:tensorflow:nll_loss = 9289.061, kl_w = 0.99983346, kl_loss = 1147.5044 (142.744 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 8253 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpqgwokls5/model.ckpt.\n",
      "INFO:tensorflow:global_step/sec: 0.663262\n",
      "INFO:tensorflow:loss = 10006.566, step = 8321 (150.770 sec)\n",
      "INFO:tensorflow:nll_loss = 8807.808, kl_w = 0.999859, kl_loss = 1198.928 (150.769 sec)\n",
      "INFO:tensorflow:global_step/sec: 0.723417\n",
      "INFO:tensorflow:loss = 10050.129, step = 8421 (138.233 sec)\n",
      "INFO:tensorflow:nll_loss = 8887.66, kl_w = 0.9998807, kl_loss = 1162.6079 (138.233 sec)\n",
      "INFO:tensorflow:global_step/sec: 0.789567\n",
      "INFO:tensorflow:loss = 10204.858, step = 8521 (126.652 sec)\n",
      "INFO:tensorflow:nll_loss = 9012.422, kl_w = 0.9998989, kl_loss = 1192.5575 (126.652 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 8602 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpqgwokls5/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 2744.4653.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpqgwokls5/model.ckpt-8602\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "\n",
      "Original: i love this film and i think it is one of the best films\n",
      "Reconstr: i really enjoyed this film i have been a <unk> for a fan of <end>\n",
      "\n",
      "Original: this movie is a waste of time and there is no point to watch it\n",
      "Reconstr: this movie is one of my favorite movies in the time to be the <end>\n",
      "\n",
      "Epoch 12/30\n",
      "\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpqgwokls5/model.ckpt-8602\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 8603 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpqgwokls5/model.ckpt.\n",
      "INFO:tensorflow:loss = 9967.024, step = 8603\n",
      "INFO:tensorflow:nll_loss = 8775.441, kl_w = 0.9999119, kl_loss = 1191.6881\n",
      "INFO:tensorflow:global_step/sec: 0.730263\n",
      "INFO:tensorflow:loss = 10786.151, step = 8703 (136.938 sec)\n",
      "INFO:tensorflow:nll_loss = 9659.494, kl_w = 0.9999254, kl_loss = 1126.7415 (136.938 sec)\n",
      "INFO:tensorflow:global_step/sec: 0.743348\n",
      "INFO:tensorflow:loss = 10684.845, step = 8803 (134.527 sec)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:nll_loss = 9509.91, kl_w = 0.9999368, kl_loss = 1175.0089 (134.527 sec)\n",
      "INFO:tensorflow:global_step/sec: 0.795414\n",
      "INFO:tensorflow:loss = 10582.662, step = 8903 (125.720 sec)\n",
      "INFO:tensorflow:nll_loss = 9437.434, kl_w = 0.9999466, kl_loss = 1145.2896 (125.719 sec)\n",
      "INFO:tensorflow:global_step/sec: 0.759166\n",
      "INFO:tensorflow:loss = 10479.792, step = 9003 (131.724 sec)\n",
      "INFO:tensorflow:nll_loss = 9346.637, kl_w = 0.9999547, kl_loss = 1133.2068 (131.724 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 9055 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpqgwokls5/model.ckpt.\n",
      "INFO:tensorflow:global_step/sec: 0.725741\n",
      "INFO:tensorflow:loss = 10373.581, step = 9103 (137.790 sec)\n",
      "INFO:tensorflow:nll_loss = 9161.928, kl_w = 0.99996173, kl_loss = 1211.6995 (137.790 sec)\n",
      "INFO:tensorflow:global_step/sec: 0.705545\n",
      "INFO:tensorflow:loss = 10183.721, step = 9203 (141.735 sec)\n",
      "INFO:tensorflow:nll_loss = 8972.045, kl_w = 0.9999676, kl_loss = 1211.7156 (141.735 sec)\n",
      "INFO:tensorflow:global_step/sec: 0.635564\n",
      "INFO:tensorflow:loss = 10240.023, step = 9303 (157.340 sec)\n",
      "INFO:tensorflow:nll_loss = 9002.771, kl_w = 0.9999726, kl_loss = 1237.2854 (157.340 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 9384 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpqgwokls5/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 2459.1497.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpqgwokls5/model.ckpt-9384\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "\n",
      "Original: i love this film and i think it is one of the best films\n",
      "Reconstr: i have seen this movie i have ever seen a huge fan of the <end>\n",
      "\n",
      "Original: this movie is a waste of time and there is no point to watch it\n",
      "Reconstr: this movie is a waste of time to watch this movie to watch it <end>\n",
      "\n",
      "Epoch 13/30\n",
      "\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpqgwokls5/model.ckpt-9384\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 9385 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpqgwokls5/model.ckpt.\n",
      "INFO:tensorflow:loss = 10432.654, step = 9385\n",
      "INFO:tensorflow:nll_loss = 9209.154, kl_w = 0.99997604, kl_loss = 1223.529\n",
      "INFO:tensorflow:global_step/sec: 0.674609\n",
      "INFO:tensorflow:loss = 10119.793, step = 9485 (148.236 sec)\n",
      "INFO:tensorflow:nll_loss = 8856.971, kl_w = 0.99997973, kl_loss = 1262.8477 (148.236 sec)\n",
      "INFO:tensorflow:global_step/sec: 0.650474\n",
      "INFO:tensorflow:loss = 9778.309, step = 9585 (153.734 sec)\n",
      "INFO:tensorflow:nll_loss = 8486.193, kl_w = 0.99998283, kl_loss = 1292.1372 (153.734 sec)\n",
      "INFO:tensorflow:global_step/sec: 0.585408\n",
      "INFO:tensorflow:loss = 9892.701, step = 9685 (170.821 sec)\n",
      "INFO:tensorflow:nll_loss = 8615.075, kl_w = 0.99998546, kl_loss = 1277.6448 (170.820 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 9760 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpqgwokls5/model.ckpt.\n",
      "INFO:tensorflow:global_step/sec: 0.579362\n",
      "INFO:tensorflow:loss = 10667.619, step = 9785 (172.604 sec)\n",
      "INFO:tensorflow:nll_loss = 9508.164, kl_w = 0.9999877, kl_loss = 1159.4695 (172.604 sec)\n",
      "INFO:tensorflow:global_step/sec: 0.658874\n",
      "INFO:tensorflow:loss = 10621.527, step = 9885 (151.774 sec)\n",
      "INFO:tensorflow:nll_loss = 9436.412, kl_w = 0.9999896, kl_loss = 1185.1272 (151.774 sec)\n",
      "INFO:tensorflow:global_step/sec: 0.747793\n",
      "INFO:tensorflow:loss = 10695.521, step = 9985 (133.726 sec)\n",
      "INFO:tensorflow:nll_loss = 9506.211, kl_w = 0.9999912, kl_loss = 1189.3206 (133.726 sec)\n",
      "INFO:tensorflow:global_step/sec: 0.786652\n",
      "INFO:tensorflow:loss = 10817.006, step = 10085 (127.121 sec)\n",
      "INFO:tensorflow:nll_loss = 9653.418, kl_w = 0.9999925, kl_loss = 1163.597 (127.121 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 10166 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpqgwokls5/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 2567.8042.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpqgwokls5/model.ckpt-10166\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "\n",
      "Original: i love this film and i think it is one of the best films\n",
      "Reconstr: i think this movie i think this movie is a huge fan of the <end>\n",
      "\n",
      "Original: this movie is a waste of time and there is no point to watch it\n",
      "Reconstr: this movie is one of those movies ever seen and the plot and the <end>\n",
      "\n",
      "Epoch 14/30\n",
      "\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpqgwokls5/model.ckpt-10166\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 10167 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpqgwokls5/model.ckpt.\n",
      "INFO:tensorflow:loss = 10066.258, step = 10167\n",
      "INFO:tensorflow:nll_loss = 8835.567, kl_w = 0.99999344, kl_loss = 1230.6985\n",
      "INFO:tensorflow:global_step/sec: 0.798067\n",
      "INFO:tensorflow:loss = 10678.28, step = 10267 (125.304 sec)\n",
      "INFO:tensorflow:nll_loss = 9477.768, kl_w = 0.9999945, kl_loss = 1200.5194 (125.304 sec)\n",
      "INFO:tensorflow:global_step/sec: 0.793811\n",
      "INFO:tensorflow:loss = 10342.213, step = 10367 (125.975 sec)\n",
      "INFO:tensorflow:nll_loss = 9111.096, kl_w = 0.99999535, kl_loss = 1231.1227 (125.975 sec)\n",
      "INFO:tensorflow:global_step/sec: 0.787566\n",
      "INFO:tensorflow:loss = 10325.605, step = 10467 (126.973 sec)\n",
      "INFO:tensorflow:nll_loss = 9097.811, kl_w = 0.99999607, kl_loss = 1227.7997 (126.973 sec)\n",
      "INFO:tensorflow:global_step/sec: 0.795946\n",
      "INFO:tensorflow:loss = 10269.962, step = 10567 (125.637 sec)\n",
      "INFO:tensorflow:nll_loss = 9012.853, kl_w = 0.99999666, kl_loss = 1257.1135 (125.637 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 10643 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpqgwokls5/model.ckpt.\n",
      "INFO:tensorflow:global_step/sec: 0.784775\n",
      "INFO:tensorflow:loss = 10213.917, step = 10667 (127.425 sec)\n",
      "INFO:tensorflow:nll_loss = 8943.837, kl_w = 0.99999714, kl_loss = 1270.0836 (127.425 sec)\n",
      "INFO:tensorflow:global_step/sec: 0.79553\n",
      "INFO:tensorflow:loss = 10106.8125, step = 10767 (125.702 sec)\n",
      "INFO:tensorflow:nll_loss = 8863.21, kl_w = 0.9999976, kl_loss = 1243.6052 (125.702 sec)\n",
      "INFO:tensorflow:global_step/sec: 0.796422\n",
      "INFO:tensorflow:loss = 10109.487, step = 10867 (125.561 sec)\n",
      "INFO:tensorflow:nll_loss = 8812.125, kl_w = 0.999998, kl_loss = 1297.3647 (125.561 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 10948 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpqgwokls5/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 2441.5999.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpqgwokls5/model.ckpt-10948\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "\n",
      "Original: i love this film and i think it is one of the best films\n",
      "Reconstr: i loved this movie when i saw this film a huge fan of <unk> <end>\n",
      "\n",
      "Original: this movie is a waste of time and there is no point to watch it\n",
      "Reconstr: this movie is one of the movie i've seen in the time to be <end>\n",
      "\n",
      "Epoch 15/30\n",
      "\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpqgwokls5/model.ckpt-10948\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 10949 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpqgwokls5/model.ckpt.\n",
      "INFO:tensorflow:loss = 10580.928, step = 10949\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:nll_loss = 9295.404, kl_w = 0.9999982, kl_loss = 1285.5261\n",
      "INFO:tensorflow:global_step/sec: 0.808415\n",
      "INFO:tensorflow:loss = 10628.579, step = 11049 (123.700 sec)\n",
      "INFO:tensorflow:nll_loss = 9387.16, kl_w = 0.99999845, kl_loss = 1241.4207 (123.700 sec)\n",
      "INFO:tensorflow:global_step/sec: 0.798896\n",
      "INFO:tensorflow:loss = 10555.951, step = 11149 (125.173 sec)\n",
      "INFO:tensorflow:nll_loss = 9339.686, kl_w = 0.9999987, kl_loss = 1216.2676 (125.173 sec)\n",
      "INFO:tensorflow:global_step/sec: 0.796561\n",
      "INFO:tensorflow:loss = 10566.429, step = 11249 (125.540 sec)\n",
      "INFO:tensorflow:nll_loss = 9316.506, kl_w = 0.9999989, kl_loss = 1249.924 (125.539 sec)\n",
      "INFO:tensorflow:global_step/sec: 0.793878\n",
      "INFO:tensorflow:loss = 9876.546, step = 11349 (125.964 sec)\n",
      "INFO:tensorflow:nll_loss = 8576.07, kl_w = 0.99999905, kl_loss = 1300.4768 (125.964 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 11427 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpqgwokls5/model.ckpt.\n",
      "INFO:tensorflow:global_step/sec: 0.785229\n",
      "INFO:tensorflow:loss = 10319.254, step = 11449 (127.351 sec)\n",
      "INFO:tensorflow:nll_loss = 9042.789, kl_w = 0.9999993, kl_loss = 1276.4657 (127.351 sec)\n",
      "INFO:tensorflow:global_step/sec: 0.797995\n",
      "INFO:tensorflow:loss = 9761.149, step = 11549 (125.314 sec)\n",
      "INFO:tensorflow:nll_loss = 8478.858, kl_w = 0.9999994, kl_loss = 1282.2915 (125.314 sec)\n",
      "INFO:tensorflow:global_step/sec: 0.798338\n",
      "INFO:tensorflow:loss = 9868.882, step = 11649 (125.260 sec)\n",
      "INFO:tensorflow:nll_loss = 8548.993, kl_w = 0.9999994, kl_loss = 1319.8893 (125.260 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 11730 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpqgwokls5/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 2562.0903.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpqgwokls5/model.ckpt-11730\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "\n",
      "Original: i love this film and i think it is one of the best films\n",
      "Reconstr: i love this movie i thought it was a huge fan of the best <end>\n",
      "\n",
      "Original: this movie is a waste of time and there is no point to watch it\n",
      "Reconstr: this movie is a lot of this movie and bad but it is a <end>\n",
      "\n",
      "Epoch 16/30\n",
      "\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpqgwokls5/model.ckpt-11730\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 11731 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpqgwokls5/model.ckpt.\n",
      "INFO:tensorflow:loss = 10511.488, step = 11731\n",
      "INFO:tensorflow:nll_loss = 9278.718, kl_w = 0.9999995, kl_loss = 1232.771\n",
      "INFO:tensorflow:global_step/sec: 0.8112\n",
      "INFO:tensorflow:loss = 10208.186, step = 11831 (123.275 sec)\n",
      "INFO:tensorflow:nll_loss = 8910.523, kl_w = 0.99999964, kl_loss = 1297.6628 (123.275 sec)\n",
      "INFO:tensorflow:global_step/sec: 0.796622\n",
      "INFO:tensorflow:loss = 9942.583, step = 11931 (125.530 sec)\n",
      "INFO:tensorflow:nll_loss = 8565.146, kl_w = 0.99999964, kl_loss = 1377.4368 (125.530 sec)\n",
      "INFO:tensorflow:global_step/sec: 0.799879\n",
      "INFO:tensorflow:loss = 10126.904, step = 12031 (125.019 sec)\n",
      "INFO:tensorflow:nll_loss = 8816.432, kl_w = 0.99999976, kl_loss = 1310.4729 (125.019 sec)\n",
      "INFO:tensorflow:global_step/sec: 0.798656\n",
      "INFO:tensorflow:loss = 10023.643, step = 12131 (125.210 sec)\n",
      "INFO:tensorflow:nll_loss = 8717.387, kl_w = 0.99999976, kl_loss = 1306.2559 (125.210 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 12211 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpqgwokls5/model.ckpt.\n",
      "INFO:tensorflow:global_step/sec: 0.788579\n",
      "INFO:tensorflow:loss = 10580.764, step = 12231 (126.810 sec)\n",
      "INFO:tensorflow:nll_loss = 9340.849, kl_w = 0.99999976, kl_loss = 1239.9153 (126.811 sec)\n",
      "INFO:tensorflow:global_step/sec: 0.798607\n",
      "INFO:tensorflow:loss = 10305.424, step = 12331 (125.218 sec)\n",
      "INFO:tensorflow:nll_loss = 9046.406, kl_w = 0.9999999, kl_loss = 1259.0172 (125.218 sec)\n",
      "INFO:tensorflow:global_step/sec: 0.797154\n",
      "INFO:tensorflow:loss = 10667.434, step = 12431 (125.446 sec)\n",
      "INFO:tensorflow:nll_loss = 9448.801, kl_w = 0.9999999, kl_loss = 1218.6333 (125.446 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 12512 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpqgwokls5/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 2626.5574.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpqgwokls5/model.ckpt-12512\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "\n",
      "Original: i love this film and i think it is one of the best films\n",
      "Reconstr: i loved this movie and i have seen this one of the worst movies <end>\n",
      "\n",
      "Original: this movie is a waste of time and there is no point to watch it\n",
      "Reconstr: this is a total waste of it to waste your time to watch it <end>\n",
      "\n",
      "Epoch 17/30\n",
      "\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpqgwokls5/model.ckpt-12512\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 12513 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpqgwokls5/model.ckpt.\n",
      "INFO:tensorflow:loss = 10287.427, step = 12513\n",
      "INFO:tensorflow:nll_loss = 8940.603, kl_w = 0.9999999, kl_loss = 1346.8242\n",
      "INFO:tensorflow:global_step/sec: 0.807622\n",
      "INFO:tensorflow:loss = 9658.63, step = 12613 (123.822 sec)\n",
      "INFO:tensorflow:nll_loss = 8321.606, kl_w = 0.9999999, kl_loss = 1337.0236 (123.822 sec)\n",
      "INFO:tensorflow:global_step/sec: 0.797196\n",
      "INFO:tensorflow:loss = 10049.81, step = 12713 (125.440 sec)\n",
      "INFO:tensorflow:nll_loss = 8690.674, kl_w = 0.9999999, kl_loss = 1359.1355 (125.440 sec)\n",
      "INFO:tensorflow:global_step/sec: 0.797087\n",
      "INFO:tensorflow:loss = 9477.0205, step = 12813 (125.457 sec)\n",
      "INFO:tensorflow:nll_loss = 8126.8228, kl_w = 0.9999999, kl_loss = 1350.198 (125.457 sec)\n",
      "INFO:tensorflow:global_step/sec: 0.796137\n",
      "INFO:tensorflow:loss = 10494.064, step = 12913 (125.607 sec)\n",
      "INFO:tensorflow:nll_loss = 9190.66, kl_w = 0.9999999, kl_loss = 1303.4043 (125.607 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 12991 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpqgwokls5/model.ckpt.\n",
      "INFO:tensorflow:global_step/sec: 0.78641\n",
      "INFO:tensorflow:loss = 10186.015, step = 13013 (127.160 sec)\n",
      "INFO:tensorflow:nll_loss = 8895.629, kl_w = 1.0, kl_loss = 1290.386 (127.160 sec)\n",
      "INFO:tensorflow:global_step/sec: 0.797186\n",
      "INFO:tensorflow:loss = 10289.133, step = 13113 (125.441 sec)\n",
      "INFO:tensorflow:nll_loss = 8980.48, kl_w = 1.0, kl_loss = 1308.652 (125.441 sec)\n",
      "INFO:tensorflow:global_step/sec: 0.798595\n",
      "INFO:tensorflow:loss = 10401.442, step = 13213 (125.220 sec)\n",
      "INFO:tensorflow:nll_loss = 9116.967, kl_w = 1.0, kl_loss = 1284.476 (125.220 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 13294 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpqgwokls5/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 2383.0657.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpqgwokls5/model.ckpt-13294\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "\n",
      "Original: i love this film and i think it is one of the best films\n",
      "Reconstr: i love this movie i think it is a huge fan of the best <end>\n",
      "\n",
      "Original: this movie is a waste of time and there is no point to watch it\n",
      "Reconstr: this movie is a lot of it to watch it and it is not <end>\n",
      "\n",
      "Epoch 18/30\n",
      "\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpqgwokls5/model.ckpt-13294\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 13295 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpqgwokls5/model.ckpt.\n",
      "INFO:tensorflow:loss = 10042.709, step = 13295\n",
      "INFO:tensorflow:nll_loss = 8669.055, kl_w = 1.0, kl_loss = 1373.6543\n",
      "INFO:tensorflow:global_step/sec: 0.813114\n",
      "INFO:tensorflow:loss = 9901.697, step = 13395 (122.985 sec)\n",
      "INFO:tensorflow:nll_loss = 8574.932, kl_w = 1.0, kl_loss = 1326.7659 (122.985 sec)\n",
      "INFO:tensorflow:global_step/sec: 0.801922\n",
      "INFO:tensorflow:loss = 10293.38, step = 13495 (124.700 sec)\n",
      "INFO:tensorflow:nll_loss = 8996.635, kl_w = 1.0, kl_loss = 1296.7451 (124.700 sec)\n",
      "INFO:tensorflow:global_step/sec: 0.802225\n",
      "INFO:tensorflow:loss = 10361.375, step = 13595 (124.653 sec)\n",
      "INFO:tensorflow:nll_loss = 9046.932, kl_w = 1.0, kl_loss = 1314.4438 (124.653 sec)\n",
      "INFO:tensorflow:global_step/sec: 0.801316\n",
      "INFO:tensorflow:loss = 10293.802, step = 13695 (124.795 sec)\n",
      "INFO:tensorflow:nll_loss = 8998.208, kl_w = 1.0, kl_loss = 1295.5939 (124.795 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 13776 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpqgwokls5/model.ckpt.\n",
      "INFO:tensorflow:global_step/sec: 0.791173\n",
      "INFO:tensorflow:loss = 10259.841, step = 13795 (126.394 sec)\n",
      "INFO:tensorflow:nll_loss = 8946.359, kl_w = 1.0, kl_loss = 1313.4814 (126.394 sec)\n",
      "INFO:tensorflow:global_step/sec: 0.800953\n",
      "INFO:tensorflow:loss = 9824.774, step = 13895 (124.851 sec)\n",
      "INFO:tensorflow:nll_loss = 8466.365, kl_w = 1.0, kl_loss = 1358.4095 (124.851 sec)\n",
      "INFO:tensorflow:global_step/sec: 0.799126\n",
      "INFO:tensorflow:loss = 10092.076, step = 13995 (125.137 sec)\n",
      "INFO:tensorflow:nll_loss = 8754.456, kl_w = 1.0, kl_loss = 1337.6204 (125.137 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 14076 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpqgwokls5/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 2387.2097.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpqgwokls5/model.ckpt-14076\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "\n",
      "Original: i love this film and i think it is one of the best films\n",
      "Reconstr: i love this movie i am a fan of a fan of the best <end>\n",
      "\n",
      "Original: this movie is a waste of time and there is no point to watch it\n",
      "Reconstr: this movie is one of those movies to watch it with a long time <end>\n",
      "\n",
      "Epoch 19/30\n",
      "\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpqgwokls5/model.ckpt-14076\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 14077 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpqgwokls5/model.ckpt.\n",
      "INFO:tensorflow:loss = 9684.898, step = 14077\n",
      "INFO:tensorflow:nll_loss = 8321.002, kl_w = 1.0, kl_loss = 1363.8967\n",
      "INFO:tensorflow:global_step/sec: 0.809171\n",
      "INFO:tensorflow:loss = 10346.23, step = 14177 (123.584 sec)\n",
      "INFO:tensorflow:nll_loss = 9016.3545, kl_w = 1.0, kl_loss = 1329.8757 (123.584 sec)\n",
      "INFO:tensorflow:global_step/sec: 0.803428\n",
      "INFO:tensorflow:loss = 10491.499, step = 14277 (124.467 sec)\n",
      "INFO:tensorflow:nll_loss = 9121.59, kl_w = 1.0, kl_loss = 1369.9094 (124.467 sec)\n",
      "INFO:tensorflow:global_step/sec: 0.799745\n",
      "INFO:tensorflow:loss = 10042.793, step = 14377 (125.040 sec)\n",
      "INFO:tensorflow:nll_loss = 8691.66, kl_w = 1.0, kl_loss = 1351.1327 (125.040 sec)\n",
      "INFO:tensorflow:global_step/sec: 0.799763\n",
      "INFO:tensorflow:loss = 10461.721, step = 14477 (125.037 sec)\n",
      "INFO:tensorflow:nll_loss = 9177.658, kl_w = 1.0, kl_loss = 1284.063 (125.037 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 14558 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpqgwokls5/model.ckpt.\n",
      "INFO:tensorflow:global_step/sec: 0.792256\n",
      "INFO:tensorflow:loss = 10196.844, step = 14577 (126.222 sec)\n",
      "INFO:tensorflow:nll_loss = 8837.717, kl_w = 1.0, kl_loss = 1359.1267 (126.222 sec)\n",
      "INFO:tensorflow:global_step/sec: 0.801835\n",
      "INFO:tensorflow:loss = 10085.453, step = 14677 (124.714 sec)\n",
      "INFO:tensorflow:nll_loss = 8713.715, kl_w = 1.0, kl_loss = 1371.7379 (124.714 sec)\n",
      "INFO:tensorflow:global_step/sec: 0.736815\n",
      "INFO:tensorflow:loss = 9806.664, step = 14777 (135.720 sec)\n",
      "INFO:tensorflow:nll_loss = 8438.617, kl_w = 1.0, kl_loss = 1368.0471 (135.720 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 14858 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpqgwokls5/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 2396.248.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpqgwokls5/model.ckpt-14858\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "\n",
      "Original: i love this film and i think it is one of the best films\n",
      "Reconstr: i love this movie to see this one of my favorite of my favorite <end>\n",
      "\n",
      "Original: this movie is a waste of time and there is no point to watch it\n",
      "Reconstr: this movie is one of those movies that it is not to watch the <end>\n",
      "\n",
      "Epoch 20/30\n",
      "\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpqgwokls5/model.ckpt-14858\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 14859 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpqgwokls5/model.ckpt.\n",
      "INFO:tensorflow:loss = 10629.838, step = 14859\n",
      "INFO:tensorflow:nll_loss = 9259.586, kl_w = 1.0, kl_loss = 1370.2516\n",
      "INFO:tensorflow:global_step/sec: 0.750987\n",
      "INFO:tensorflow:loss = 10308.075, step = 14959 (133.159 sec)\n",
      "INFO:tensorflow:nll_loss = 8982.26, kl_w = 1.0, kl_loss = 1325.8156 (133.159 sec)\n",
      "INFO:tensorflow:global_step/sec: 0.724682\n",
      "INFO:tensorflow:loss = 10503.024, step = 15059 (137.991 sec)\n",
      "INFO:tensorflow:nll_loss = 9173.357, kl_w = 1.0, kl_loss = 1329.6667 (137.991 sec)\n",
      "INFO:tensorflow:global_step/sec: 0.760171\n",
      "INFO:tensorflow:loss = 9975.016, step = 15159 (131.549 sec)\n",
      "INFO:tensorflow:nll_loss = 8549.566, kl_w = 1.0, kl_loss = 1425.4489 (131.549 sec)\n",
      "INFO:tensorflow:global_step/sec: 0.787364\n",
      "INFO:tensorflow:loss = 9979.131, step = 15259 (127.006 sec)\n",
      "INFO:tensorflow:nll_loss = 8571.625, kl_w = 1.0, kl_loss = 1407.5061 (127.006 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 15314 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpqgwokls5/model.ckpt.\n",
      "INFO:tensorflow:global_step/sec: 0.78699\n",
      "INFO:tensorflow:loss = 9710.493, step = 15359 (127.066 sec)\n",
      "INFO:tensorflow:nll_loss = 8309.32, kl_w = 1.0, kl_loss = 1401.1731 (127.066 sec)\n",
      "INFO:tensorflow:global_step/sec: 0.799087\n",
      "INFO:tensorflow:loss = 9985.519, step = 15459 (125.143 sec)\n",
      "INFO:tensorflow:nll_loss = 8548.071, kl_w = 1.0, kl_loss = 1437.4469 (125.143 sec)\n",
      "INFO:tensorflow:global_step/sec: 0.800695\n",
      "INFO:tensorflow:loss = 10439.867, step = 15559 (124.892 sec)\n",
      "INFO:tensorflow:nll_loss = 9110.905, kl_w = 1.0, kl_loss = 1328.9619 (124.892 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 15640 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpqgwokls5/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 2545.456.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpqgwokls5/model.ckpt-15640\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "\n",
      "Original: i love this film and i think it is one of the best films\n",
      "Reconstr: i love this movie and i think it is one of the best films <end>\n",
      "\n",
      "Original: this movie is a waste of time and there is no point to watch it\n",
      "Reconstr: this movie is a waste of time and not waste your time to be <end>\n",
      "\n",
      "Epoch 21/30\n",
      "\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpqgwokls5/model.ckpt-15640\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 15641 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpqgwokls5/model.ckpt.\n",
      "INFO:tensorflow:loss = 10558.953, step = 15641\n",
      "INFO:tensorflow:nll_loss = 9237.1455, kl_w = 1.0, kl_loss = 1321.8071\n",
      "INFO:tensorflow:global_step/sec: 0.812453\n",
      "INFO:tensorflow:loss = 10064.067, step = 15741 (123.085 sec)\n",
      "INFO:tensorflow:nll_loss = 8726.362, kl_w = 1.0, kl_loss = 1337.7051 (123.085 sec)\n",
      "INFO:tensorflow:global_step/sec: 0.80349\n",
      "INFO:tensorflow:loss = 10009.711, step = 15841 (124.457 sec)\n",
      "INFO:tensorflow:nll_loss = 8604.828, kl_w = 1.0, kl_loss = 1404.883 (124.457 sec)\n",
      "INFO:tensorflow:global_step/sec: 0.801593\n",
      "INFO:tensorflow:loss = 9671.873, step = 15941 (124.752 sec)\n",
      "INFO:tensorflow:nll_loss = 8246.181, kl_w = 1.0, kl_loss = 1425.6919 (124.751 sec)\n",
      "INFO:tensorflow:global_step/sec: 0.804616\n",
      "INFO:tensorflow:loss = 9769.299, step = 16041 (124.283 sec)\n",
      "INFO:tensorflow:nll_loss = 8316.494, kl_w = 1.0, kl_loss = 1452.8047 (124.283 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 16123 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpqgwokls5/model.ckpt.\n",
      "INFO:tensorflow:global_step/sec: 0.794738\n",
      "INFO:tensorflow:loss = 9849.473, step = 16141 (125.828 sec)\n",
      "INFO:tensorflow:nll_loss = 8433.575, kl_w = 1.0, kl_loss = 1415.898 (125.828 sec)\n",
      "INFO:tensorflow:global_step/sec: 0.808752\n",
      "INFO:tensorflow:loss = 9987.4795, step = 16241 (123.647 sec)\n",
      "INFO:tensorflow:nll_loss = 8602.217, kl_w = 1.0, kl_loss = 1385.2623 (123.647 sec)\n",
      "INFO:tensorflow:global_step/sec: 0.790144\n",
      "INFO:tensorflow:loss = 10281.307, step = 16341 (126.559 sec)\n",
      "INFO:tensorflow:nll_loss = 8906.106, kl_w = 1.0, kl_loss = 1375.1997 (126.559 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 16422 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpqgwokls5/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 2526.484.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpqgwokls5/model.ckpt-16422\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "\n",
      "Original: i love this film and i think it is one of the best films\n",
      "Reconstr: i really enjoyed this movie i would have been a lot of my favorite <end>\n",
      "\n",
      "Original: this movie is a waste of time and there is no point to watch it\n",
      "Reconstr: this is one of the worst movies ever seen it is not to watch <end>\n",
      "\n",
      "Epoch 22/30\n",
      "\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpqgwokls5/model.ckpt-16422\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 16423 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpqgwokls5/model.ckpt.\n",
      "INFO:tensorflow:loss = 10093.638, step = 16423\n",
      "INFO:tensorflow:nll_loss = 8652.291, kl_w = 1.0, kl_loss = 1441.3467\n",
      "INFO:tensorflow:global_step/sec: 0.725002\n",
      "INFO:tensorflow:loss = 9801.043, step = 16523 (137.932 sec)\n",
      "INFO:tensorflow:nll_loss = 8355.567, kl_w = 1.0, kl_loss = 1445.4757 (137.932 sec)\n",
      "INFO:tensorflow:global_step/sec: 0.72543\n",
      "INFO:tensorflow:loss = 9737.68, step = 16623 (137.850 sec)\n",
      "INFO:tensorflow:nll_loss = 8347.265, kl_w = 1.0, kl_loss = 1390.4152 (137.850 sec)\n",
      "INFO:tensorflow:global_step/sec: 0.715435\n",
      "INFO:tensorflow:loss = 10153.812, step = 16723 (139.775 sec)\n",
      "INFO:tensorflow:nll_loss = 8751.44, kl_w = 1.0, kl_loss = 1402.3713 (139.774 sec)\n",
      "INFO:tensorflow:global_step/sec: 0.706713\n",
      "INFO:tensorflow:loss = 10337.918, step = 16823 (141.500 sec)\n",
      "INFO:tensorflow:nll_loss = 8959.059, kl_w = 1.0, kl_loss = 1378.8594 (141.500 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 16852 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpqgwokls5/model.ckpt.\n",
      "INFO:tensorflow:global_step/sec: 0.689227\n",
      "INFO:tensorflow:loss = 10341.615, step = 16923 (145.090 sec)\n",
      "INFO:tensorflow:nll_loss = 8978.605, kl_w = 1.0, kl_loss = 1363.0099 (145.090 sec)\n",
      "INFO:tensorflow:global_step/sec: 0.73845\n",
      "INFO:tensorflow:loss = 10211.037, step = 17023 (135.419 sec)\n",
      "INFO:tensorflow:nll_loss = 8810.193, kl_w = 1.0, kl_loss = 1400.8433 (135.419 sec)\n",
      "INFO:tensorflow:global_step/sec: 0.720295\n",
      "INFO:tensorflow:loss = 9811.671, step = 17123 (138.832 sec)\n",
      "INFO:tensorflow:nll_loss = 8367.465, kl_w = 1.0, kl_loss = 1444.2058 (138.832 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 17204 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpqgwokls5/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 2489.141.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpqgwokls5/model.ckpt-17204\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "\n",
      "Original: i love this film and i think it is one of the best films\n",
      "Reconstr: i loved this movie and i have seen this one of the best films <end>\n",
      "\n",
      "Original: this movie is a waste of time and there is no point to watch it\n",
      "Reconstr: this movie is one of those movies that it is bad if you are <end>\n",
      "\n",
      "Epoch 23/30\n",
      "\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpqgwokls5/model.ckpt-17204\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 17205 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpqgwokls5/model.ckpt.\n",
      "INFO:tensorflow:loss = 9607.393, step = 17205\n",
      "INFO:tensorflow:nll_loss = 8136.272, kl_w = 1.0, kl_loss = 1471.1207\n",
      "INFO:tensorflow:global_step/sec: 0.737686\n",
      "INFO:tensorflow:loss = 9973.019, step = 17305 (135.560 sec)\n",
      "INFO:tensorflow:nll_loss = 8532.029, kl_w = 1.0, kl_loss = 1440.9893 (135.560 sec)\n",
      "INFO:tensorflow:global_step/sec: 0.703835\n",
      "INFO:tensorflow:loss = 10105.778, step = 17405 (142.079 sec)\n",
      "INFO:tensorflow:nll_loss = 8709.092, kl_w = 1.0, kl_loss = 1396.6868 (142.080 sec)\n",
      "INFO:tensorflow:global_step/sec: 0.726065\n",
      "INFO:tensorflow:loss = 10435.27, step = 17505 (137.728 sec)\n",
      "INFO:tensorflow:nll_loss = 9042.127, kl_w = 1.0, kl_loss = 1393.1423 (137.728 sec)\n",
      "INFO:tensorflow:global_step/sec: 0.716657\n",
      "INFO:tensorflow:loss = 10236.041, step = 17605 (139.537 sec)\n",
      "INFO:tensorflow:nll_loss = 8863.45, kl_w = 1.0, kl_loss = 1372.5907 (139.537 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 17636 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpqgwokls5/model.ckpt.\n",
      "INFO:tensorflow:global_step/sec: 0.708903\n",
      "INFO:tensorflow:loss = 10327.539, step = 17705 (141.063 sec)\n",
      "INFO:tensorflow:nll_loss = 8946.583, kl_w = 1.0, kl_loss = 1380.9563 (141.063 sec)\n",
      "INFO:tensorflow:global_step/sec: 0.682736\n",
      "INFO:tensorflow:loss = 10057.184, step = 17805 (146.469 sec)\n",
      "INFO:tensorflow:nll_loss = 8618.291, kl_w = 1.0, kl_loss = 1438.8926 (146.469 sec)\n",
      "INFO:tensorflow:global_step/sec: 0.639007\n",
      "INFO:tensorflow:loss = 9561.999, step = 17905 (156.493 sec)\n",
      "INFO:tensorflow:nll_loss = 8080.132, kl_w = 1.0, kl_loss = 1481.8673 (156.493 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 17986 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpqgwokls5/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 2558.6821.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpqgwokls5/model.ckpt-17986\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "\n",
      "Original: i love this film and i think it is one of the best films\n",
      "Reconstr: i watched this movie and i was not be a fan of my favorite <end>\n",
      "\n",
      "Original: this movie is a waste of time and there is no point to watch it\n",
      "Reconstr: this is one of those movies that you have no idea to watch it <end>\n",
      "\n",
      "Epoch 24/30\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpqgwokls5/model.ckpt-17986\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 17987 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpqgwokls5/model.ckpt.\n",
      "INFO:tensorflow:loss = 9669.643, step = 17987\n",
      "INFO:tensorflow:nll_loss = 8199.959, kl_w = 1.0, kl_loss = 1469.6836\n",
      "INFO:tensorflow:global_step/sec: 0.736541\n",
      "INFO:tensorflow:loss = 9624.725, step = 18087 (135.771 sec)\n",
      "INFO:tensorflow:nll_loss = 8155.2695, kl_w = 1.0, kl_loss = 1469.4547 (135.771 sec)\n",
      "INFO:tensorflow:global_step/sec: 0.697171\n",
      "INFO:tensorflow:loss = 9413.46, step = 18187 (143.437 sec)\n",
      "INFO:tensorflow:nll_loss = 7940.933, kl_w = 1.0, kl_loss = 1472.5267 (143.437 sec)\n",
      "INFO:tensorflow:global_step/sec: 0.664737\n",
      "INFO:tensorflow:loss = 10225.074, step = 18287 (150.435 sec)\n",
      "INFO:tensorflow:nll_loss = 8834.438, kl_w = 1.0, kl_loss = 1390.6362 (150.435 sec)\n",
      "INFO:tensorflow:global_step/sec: 0.649067\n",
      "INFO:tensorflow:loss = 10326.64, step = 18387 (154.069 sec)\n",
      "INFO:tensorflow:nll_loss = 8941.978, kl_w = 1.0, kl_loss = 1384.6619 (154.069 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 18396 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpqgwokls5/model.ckpt.\n",
      "INFO:tensorflow:global_step/sec: 0.636086\n",
      "INFO:tensorflow:loss = 10320.482, step = 18487 (157.210 sec)\n",
      "INFO:tensorflow:nll_loss = 8963.905, kl_w = 1.0, kl_loss = 1356.577 (157.209 sec)\n",
      "INFO:tensorflow:global_step/sec: 0.616537\n",
      "INFO:tensorflow:loss = 10342.56, step = 18587 (162.199 sec)\n",
      "INFO:tensorflow:nll_loss = 8963.057, kl_w = 1.0, kl_loss = 1379.5029 (162.200 sec)\n",
      "INFO:tensorflow:global_step/sec: 0.606499\n",
      "INFO:tensorflow:loss = 9819.451, step = 18687 (164.878 sec)\n",
      "INFO:tensorflow:nll_loss = 8359.616, kl_w = 1.0, kl_loss = 1459.835 (164.877 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 18768 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpqgwokls5/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 2399.603.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpqgwokls5/model.ckpt-18768\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "\n",
      "Original: i love this film and i think it is one of the best films\n",
      "Reconstr: i loved this movie and i have seen this one of the worst movies <end>\n",
      "\n",
      "Original: this movie is a waste of time and there is no point to watch it\n",
      "Reconstr: this movie is a waste of 10 for a long time to watch it <end>\n",
      "\n",
      "Epoch 25/30\n",
      "\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpqgwokls5/model.ckpt-18768\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 18769 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpqgwokls5/model.ckpt.\n",
      "INFO:tensorflow:loss = 9560.815, step = 18769\n",
      "INFO:tensorflow:nll_loss = 8025.726, kl_w = 1.0, kl_loss = 1535.0896\n",
      "INFO:tensorflow:global_step/sec: 0.63499\n",
      "INFO:tensorflow:loss = 9664.998, step = 18869 (157.484 sec)\n",
      "INFO:tensorflow:nll_loss = 8186.7637, kl_w = 1.0, kl_loss = 1478.2349 (157.484 sec)\n",
      "INFO:tensorflow:global_step/sec: 0.648776\n",
      "INFO:tensorflow:loss = 10131.959, step = 18969 (154.137 sec)\n",
      "INFO:tensorflow:nll_loss = 8709.418, kl_w = 1.0, kl_loss = 1422.541 (154.137 sec)\n",
      "INFO:tensorflow:global_step/sec: 0.618808\n",
      "INFO:tensorflow:loss = 10257.201, step = 19069 (161.601 sec)\n",
      "INFO:tensorflow:nll_loss = 8860.572, kl_w = 1.0, kl_loss = 1396.6287 (161.601 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 19146 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpqgwokls5/model.ckpt.\n",
      "INFO:tensorflow:global_step/sec: 0.603178\n",
      "INFO:tensorflow:loss = 10172.757, step = 19169 (165.790 sec)\n",
      "INFO:tensorflow:nll_loss = 8700.498, kl_w = 1.0, kl_loss = 1472.2589 (165.791 sec)\n",
      "INFO:tensorflow:global_step/sec: 0.627256\n",
      "INFO:tensorflow:loss = 10213.724, step = 19269 (159.424 sec)\n",
      "INFO:tensorflow:nll_loss = 8806.159, kl_w = 1.0, kl_loss = 1407.5645 (159.423 sec)\n",
      "INFO:tensorflow:global_step/sec: 0.660451\n",
      "INFO:tensorflow:loss = 9793.187, step = 19369 (151.411 sec)\n",
      "INFO:tensorflow:nll_loss = 8325.579, kl_w = 1.0, kl_loss = 1467.6074 (151.413 sec)\n",
      "INFO:tensorflow:global_step/sec: 0.672166\n",
      "INFO:tensorflow:loss = 9824.51, step = 19469 (148.773 sec)\n",
      "INFO:tensorflow:nll_loss = 8307.105, kl_w = 1.0, kl_loss = 1517.4045 (148.771 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 19540 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpqgwokls5/model.ckpt.\n",
      "INFO:tensorflow:Saving checkpoints for 19550 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpqgwokls5/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 2469.588.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpqgwokls5/model.ckpt-19550\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "\n",
      "Original: i love this film and i think it is one of the best films\n",
      "Reconstr: i saw this movie and it is to be one of the worst films <end>\n",
      "\n",
      "Original: this movie is a waste of time and there is no point to watch it\n",
      "Reconstr: this movie is one of those movies like to watch movies that you are <end>\n",
      "\n",
      "Epoch 26/30\n",
      "\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpqgwokls5/model.ckpt-19550\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 19551 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpqgwokls5/model.ckpt.\n",
      "INFO:tensorflow:loss = 10155.067, step = 19551\n",
      "INFO:tensorflow:nll_loss = 8715.861, kl_w = 1.0, kl_loss = 1439.2058\n",
      "INFO:tensorflow:global_step/sec: 0.689537\n",
      "INFO:tensorflow:loss = 10164.33, step = 19651 (145.026 sec)\n",
      "INFO:tensorflow:nll_loss = 8779.092, kl_w = 1.0, kl_loss = 1385.2378 (145.026 sec)\n",
      "INFO:tensorflow:global_step/sec: 0.702371\n",
      "INFO:tensorflow:loss = 9710.241, step = 19751 (142.376 sec)\n",
      "INFO:tensorflow:nll_loss = 8231.936, kl_w = 1.0, kl_loss = 1478.3054 (142.377 sec)\n",
      "INFO:tensorflow:global_step/sec: 0.731766\n",
      "INFO:tensorflow:loss = 9524.54, step = 19851 (136.654 sec)\n",
      "INFO:tensorflow:nll_loss = 7998.966, kl_w = 1.0, kl_loss = 1525.5742 (136.653 sec)\n",
      "INFO:tensorflow:global_step/sec: 0.685741\n",
      "INFO:tensorflow:loss = 9691.219, step = 19951 (145.828 sec)\n",
      "INFO:tensorflow:nll_loss = 8189.0596, kl_w = 1.0, kl_loss = 1502.1597 (145.828 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 19972 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpqgwokls5/model.ckpt.\n",
      "INFO:tensorflow:global_step/sec: 0.718105\n",
      "INFO:tensorflow:loss = 9461.453, step = 20051 (139.255 sec)\n",
      "INFO:tensorflow:nll_loss = 7925.038, kl_w = 1.0, kl_loss = 1536.4154 (139.255 sec)\n",
      "INFO:tensorflow:global_step/sec: 0.680882\n",
      "INFO:tensorflow:loss = 10197.28, step = 20151 (146.868 sec)\n",
      "INFO:tensorflow:nll_loss = 8759.175, kl_w = 1.0, kl_loss = 1438.1052 (146.869 sec)\n",
      "INFO:tensorflow:global_step/sec: 0.661962\n",
      "INFO:tensorflow:loss = 10459.025, step = 20251 (151.066 sec)\n",
      "INFO:tensorflow:nll_loss = 9007.205, kl_w = 1.0, kl_loss = 1451.8204 (151.066 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 20332 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpqgwokls5/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 2489.6516.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpqgwokls5/model.ckpt-20332\n",
      "INFO:tensorflow:Running local_init_op.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Done running local_init_op.\n",
      "\n",
      "Original: i love this film and i think it is one of the best films\n",
      "Reconstr: i loved this movie and i first saw it on the top of the <end>\n",
      "\n",
      "Original: this movie is a waste of time and there is no point to watch it\n",
      "Reconstr: this movie is one of the best i've seen in all time to the <end>\n",
      "\n",
      "Epoch 27/30\n",
      "\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpqgwokls5/model.ckpt-20332\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 20333 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpqgwokls5/model.ckpt.\n",
      "INFO:tensorflow:loss = 9808.472, step = 20333\n",
      "INFO:tensorflow:nll_loss = 8283.312, kl_w = 1.0, kl_loss = 1525.1604\n",
      "INFO:tensorflow:global_step/sec: 0.620099\n",
      "INFO:tensorflow:loss = 10006.894, step = 20433 (161.268 sec)\n",
      "INFO:tensorflow:nll_loss = 8493.202, kl_w = 1.0, kl_loss = 1513.6917 (161.267 sec)\n",
      "INFO:tensorflow:global_step/sec: 0.614125\n",
      "INFO:tensorflow:loss = 10113.816, step = 20533 (162.831 sec)\n",
      "INFO:tensorflow:nll_loss = 8705.096, kl_w = 1.0, kl_loss = 1408.721 (162.831 sec)\n",
      "INFO:tensorflow:global_step/sec: 0.622595\n",
      "INFO:tensorflow:loss = 10068.262, step = 20633 (160.618 sec)\n",
      "INFO:tensorflow:nll_loss = 8579.604, kl_w = 1.0, kl_loss = 1488.658 (160.618 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 20705 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpqgwokls5/model.ckpt.\n",
      "INFO:tensorflow:global_step/sec: 0.629207\n",
      "INFO:tensorflow:loss = 10147.102, step = 20733 (158.933 sec)\n",
      "INFO:tensorflow:nll_loss = 8728.696, kl_w = 1.0, kl_loss = 1418.4049 (158.933 sec)\n",
      "INFO:tensorflow:global_step/sec: 0.645396\n",
      "INFO:tensorflow:loss = 9967.361, step = 20833 (154.941 sec)\n",
      "INFO:tensorflow:nll_loss = 8477.65, kl_w = 1.0, kl_loss = 1489.7104 (154.941 sec)\n",
      "INFO:tensorflow:global_step/sec: 0.670505\n",
      "INFO:tensorflow:loss = 9742.996, step = 20933 (149.141 sec)\n",
      "INFO:tensorflow:nll_loss = 8233.746, kl_w = 1.0, kl_loss = 1509.2498 (149.142 sec)\n",
      "INFO:tensorflow:global_step/sec: 0.684471\n",
      "INFO:tensorflow:loss = 9624.965, step = 21033 (146.098 sec)\n",
      "INFO:tensorflow:nll_loss = 8087.586, kl_w = 1.0, kl_loss = 1537.3794 (146.098 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 21109 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpqgwokls5/model.ckpt.\n",
      "INFO:tensorflow:Saving checkpoints for 21114 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpqgwokls5/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 2448.6538.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpqgwokls5/model.ckpt-21114\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "\n",
      "Original: i love this film and i think it is one of the best films\n",
      "Reconstr: i love this movie and i love it as a fan of the best <end>\n",
      "\n",
      "Original: this movie is a waste of time and there is no point to watch it\n",
      "Reconstr: this is a very low budget movie about the time i don't understand why <end>\n",
      "\n",
      "Epoch 28/30\n",
      "\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpqgwokls5/model.ckpt-21114\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 21115 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpqgwokls5/model.ckpt.\n",
      "INFO:tensorflow:loss = 9348.415, step = 21115\n",
      "INFO:tensorflow:nll_loss = 7789.8076, kl_w = 1.0, kl_loss = 1558.607\n",
      "INFO:tensorflow:global_step/sec: 0.703375\n",
      "INFO:tensorflow:loss = 10216.637, step = 21215 (142.173 sec)\n",
      "INFO:tensorflow:nll_loss = 8796.379, kl_w = 1.0, kl_loss = 1420.2577 (142.172 sec)\n",
      "INFO:tensorflow:global_step/sec: 0.695902\n",
      "INFO:tensorflow:loss = 10185.808, step = 21315 (143.699 sec)\n",
      "INFO:tensorflow:nll_loss = 8708.763, kl_w = 1.0, kl_loss = 1477.0452 (143.699 sec)\n",
      "INFO:tensorflow:global_step/sec: 0.765827\n",
      "INFO:tensorflow:loss = 10240.6455, step = 21415 (130.577 sec)\n",
      "INFO:tensorflow:nll_loss = 8817.612, kl_w = 1.0, kl_loss = 1423.033 (130.577 sec)\n",
      "INFO:tensorflow:global_step/sec: 0.740827\n",
      "INFO:tensorflow:loss = 10166.758, step = 21515 (134.984 sec)\n",
      "INFO:tensorflow:nll_loss = 8762.145, kl_w = 1.0, kl_loss = 1404.6134 (134.984 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 21548 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpqgwokls5/model.ckpt.\n",
      "INFO:tensorflow:global_step/sec: 0.699824\n",
      "INFO:tensorflow:loss = 9617.366, step = 21615 (142.893 sec)\n",
      "INFO:tensorflow:nll_loss = 8084.457, kl_w = 1.0, kl_loss = 1532.9095 (142.893 sec)\n",
      "INFO:tensorflow:global_step/sec: 0.720931\n",
      "INFO:tensorflow:loss = 9545.626, step = 21715 (138.710 sec)\n",
      "INFO:tensorflow:nll_loss = 8008.2725, kl_w = 1.0, kl_loss = 1537.3531 (138.710 sec)\n",
      "INFO:tensorflow:global_step/sec: 0.723824\n",
      "INFO:tensorflow:loss = 9717.145, step = 21815 (138.155 sec)\n",
      "INFO:tensorflow:nll_loss = 8204.082, kl_w = 1.0, kl_loss = 1513.0625 (138.155 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 21896 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpqgwokls5/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 2512.4756.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpqgwokls5/model.ckpt-21896\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "\n",
      "Original: i love this film and i think it is one of the best films\n",
      "Reconstr: i watched this movie and i am not a fan of the best movies <end>\n",
      "\n",
      "Original: this movie is a waste of time and there is no point to watch it\n",
      "Reconstr: this is one of those movies that it is the only reason to watch <end>\n",
      "\n",
      "Epoch 29/30\n",
      "\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpqgwokls5/model.ckpt-21896\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 21897 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpqgwokls5/model.ckpt.\n",
      "INFO:tensorflow:loss = 9500.521, step = 21897\n",
      "INFO:tensorflow:nll_loss = 7992.256, kl_w = 1.0, kl_loss = 1508.2653\n",
      "INFO:tensorflow:global_step/sec: 0.696604\n",
      "INFO:tensorflow:loss = 9465.624, step = 21997 (143.555 sec)\n",
      "INFO:tensorflow:nll_loss = 7931.204, kl_w = 1.0, kl_loss = 1534.4197 (143.555 sec)\n",
      "INFO:tensorflow:global_step/sec: 0.60797\n",
      "INFO:tensorflow:loss = 10206.203, step = 22097 (164.482 sec)\n",
      "INFO:tensorflow:nll_loss = 8751.778, kl_w = 1.0, kl_loss = 1454.4243 (164.482 sec)\n",
      "INFO:tensorflow:global_step/sec: 0.653247\n",
      "INFO:tensorflow:loss = 10144.15, step = 22197 (153.081 sec)\n",
      "INFO:tensorflow:nll_loss = 8640.273, kl_w = 1.0, kl_loss = 1503.877 (153.082 sec)\n",
      "INFO:tensorflow:global_step/sec: 0.73932\n",
      "INFO:tensorflow:loss = 10209.174, step = 22297 (135.259 sec)\n",
      "INFO:tensorflow:nll_loss = 8732.02, kl_w = 1.0, kl_loss = 1477.1548 (135.259 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 22299 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpqgwokls5/model.ckpt.\n",
      "INFO:tensorflow:global_step/sec: 0.714529\n",
      "INFO:tensorflow:loss = 10228.597, step = 22397 (139.952 sec)\n",
      "INFO:tensorflow:nll_loss = 8762.209, kl_w = 1.0, kl_loss = 1466.3873 (139.953 sec)\n",
      "INFO:tensorflow:global_step/sec: 0.734946\n",
      "INFO:tensorflow:loss = 9836.642, step = 22497 (136.064 sec)\n",
      "INFO:tensorflow:nll_loss = 8314.353, kl_w = 1.0, kl_loss = 1522.2888 (136.064 sec)\n",
      "INFO:tensorflow:global_step/sec: 0.727797\n",
      "INFO:tensorflow:loss = 9493.01, step = 22597 (137.401 sec)\n",
      "INFO:tensorflow:nll_loss = 7894.931, kl_w = 1.0, kl_loss = 1598.0784 (137.401 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 22678 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpqgwokls5/model.ckpt.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loss for final step: 2530.2808.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpqgwokls5/model.ckpt-22678\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "\n",
      "Original: i love this film and i think it is one of the best films\n",
      "Reconstr: i loved this movie but i thought it was a lot of the worst <end>\n",
      "\n",
      "Original: this movie is a waste of time and there is no point to watch it\n",
      "Reconstr: this is a complete waste of time but it's a hard to watch it <end>\n",
      "\n",
      "Epoch 30/30\n",
      "\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpqgwokls5/model.ckpt-22678\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 22679 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpqgwokls5/model.ckpt.\n",
      "INFO:tensorflow:loss = 9573.744, step = 22679\n",
      "INFO:tensorflow:nll_loss = 8035.615, kl_w = 1.0, kl_loss = 1538.1292\n",
      "INFO:tensorflow:global_step/sec: 0.742702\n",
      "INFO:tensorflow:loss = 10456.567, step = 22779 (134.645 sec)\n",
      "INFO:tensorflow:nll_loss = 8985.897, kl_w = 1.0, kl_loss = 1470.6696 (134.645 sec)\n",
      "INFO:tensorflow:global_step/sec: 0.794266\n",
      "INFO:tensorflow:loss = 10358.664, step = 22879 (125.902 sec)\n",
      "INFO:tensorflow:nll_loss = 8880.57, kl_w = 1.0, kl_loss = 1478.0941 (125.902 sec)\n",
      "INFO:tensorflow:global_step/sec: 0.71564\n",
      "INFO:tensorflow:loss = 10200.568, step = 22979 (139.735 sec)\n",
      "INFO:tensorflow:nll_loss = 8735.783, kl_w = 1.0, kl_loss = 1464.7854 (139.735 sec)\n",
      "INFO:tensorflow:global_step/sec: 0.69306\n",
      "INFO:tensorflow:loss = 9857.747, step = 23079 (144.288 sec)\n",
      "INFO:tensorflow:nll_loss = 8372.248, kl_w = 1.0, kl_loss = 1485.4988 (144.288 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 23116 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpqgwokls5/model.ckpt.\n",
      "INFO:tensorflow:global_step/sec: 0.665504\n",
      "INFO:tensorflow:loss = 9637.107, step = 23179 (150.262 sec)\n",
      "INFO:tensorflow:nll_loss = 8065.122, kl_w = 1.0, kl_loss = 1571.9855 (150.262 sec)\n",
      "INFO:tensorflow:global_step/sec: 0.663468\n",
      "INFO:tensorflow:loss = 9691.427, step = 23279 (150.723 sec)\n",
      "INFO:tensorflow:nll_loss = 8196.049, kl_w = 1.0, kl_loss = 1495.3777 (150.723 sec)\n",
      "INFO:tensorflow:global_step/sec: 0.696406\n",
      "INFO:tensorflow:loss = 9685.892, step = 23379 (143.596 sec)\n",
      "INFO:tensorflow:nll_loss = 8133.3013, kl_w = 1.0, kl_loss = 1552.5906 (143.597 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 23460 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpqgwokls5/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 2504.5603.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpqgwokls5/model.ckpt-23460\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "\n",
      "Original: i love this film and i think it is one of the best films\n",
      "Reconstr: i loved this movie and i was it was a huge fan of the <end>\n",
      "\n",
      "Original: this movie is a waste of time and there is no point to watch it\n",
      "Reconstr: this movie is one of those movies if you like this movie it is <end>\n",
      "\n"
     ]
    }
   ],
   "source": [
    "main()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
